{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Labeling answers\n",
    "Given a plain text, we're going to ouput the words from it we consider to be answer-worthy.\n",
    "\n",
    "We'll first extract the words and their features, from the new text, as we did in *Feature Engineering*, then we'll one-hot encode them and put use our predictor from *Training*."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Common imports \n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pickling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import _pickle as cPickle\n",
    "from pathlib import Path\n",
    "\n",
    "def dumpPickle(fileName, content):\n",
    "    pickleFile = open(fileName, 'wb')\n",
    "    cPickle.dump(content, pickleFile, -1)\n",
    "    pickleFile.close()\n",
    "\n",
    "def loadPickle(fileName):    \n",
    "    file = open(fileName, 'rb')\n",
    "    content = cPickle.load(file)\n",
    "    file.close()\n",
    "    \n",
    "    return content\n",
    "    \n",
    "def pickleExists(fileName):\n",
    "    file = Path(fileName)\n",
    "    \n",
    "    if file.is_file():\n",
    "        return True\n",
    "    \n",
    "    return False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extracting words and their features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "from spacy import displacy\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "\n",
    "#There seems to be a bug with spacy's stop words.\n",
    "from spacy.lang.en.stop_words import STOP_WORDS\n",
    "for word in STOP_WORDS:\n",
    "    for w in (word, word[0].capitalize(), word.upper()):\n",
    "        lex = nlp.vocab[w]\n",
    "        lex.is_stop = True\n",
    "        \n",
    "#Extract answers and the sentence they are in\n",
    "def extractAnswers(qas, doc):\n",
    "    answers = []\n",
    "\n",
    "    senStart = 0\n",
    "    senId = 0\n",
    "\n",
    "    for sentence in doc.sents:\n",
    "        senLen = len(sentence.text)\n",
    "\n",
    "        for answer in qas:\n",
    "            answerStart = answer['answers'][0]['answer_start']\n",
    "\n",
    "            if (answerStart >= senStart and answerStart < (senStart + senLen)):\n",
    "                answers.append({'sentenceId': senId, 'text': answer['answers'][0]['text']})\n",
    "\n",
    "        senStart += senLen\n",
    "        senId += 1\n",
    "    \n",
    "    return answers\n",
    "\n",
    "#TODO - Clean answers from stopwords?\n",
    "def tokenIsAnswer(token, sentenceId, answers):\n",
    "    for i in range(len(answers)):\n",
    "        if (answers[i]['sentenceId'] == sentenceId):\n",
    "            if (answers[i]['text'] == token):\n",
    "                return True\n",
    "    return False\n",
    "\n",
    "#Save named entities start points\n",
    "\n",
    "def getNEStartIndexs(doc):\n",
    "    neStarts = {}\n",
    "    for ne in doc.ents:\n",
    "        neStarts[ne.start] = ne\n",
    "        \n",
    "    return neStarts \n",
    "\n",
    "def getSentenceStartIndexes(doc):\n",
    "    senStarts = []\n",
    "    \n",
    "    for sentence in doc.sents:\n",
    "        senStarts.append(sentence[0].i)\n",
    "    \n",
    "    return senStarts\n",
    "    \n",
    "def getSentenceForWordPosition(wordPos, senStarts):\n",
    "    for i in range(1, len(senStarts)):\n",
    "        if (wordPos < senStarts[i]):\n",
    "            return i - 1\n",
    "        \n",
    "def addWordsForParagrapgh(newWords, text):\n",
    "    doc = nlp(text)\n",
    "\n",
    "    neStarts = getNEStartIndexs(doc)\n",
    "    senStarts = getSentenceStartIndexes(doc)\n",
    "    \n",
    "    #index of word in spacy doc text\n",
    "    i = 0\n",
    "    \n",
    "    while (i < len(doc)):\n",
    "        #If the token is a start of a Named Entity, add it and push to index to end of the NE\n",
    "        if (i in neStarts):\n",
    "            word = neStarts[i]\n",
    "            #add word\n",
    "            currentSentence = getSentenceForWordPosition(word.start, senStarts)\n",
    "            wordLen = word.end - word.start\n",
    "            shape = ''\n",
    "            for wordIndex in range(word.start, word.end):\n",
    "                shape += (' ' + doc[wordIndex].shape_)\n",
    "\n",
    "            newWords.append([word.text,\n",
    "                            0,\n",
    "                            0,\n",
    "                            currentSentence,\n",
    "                            wordLen,\n",
    "                            word.label_,\n",
    "                            None,\n",
    "                            None,\n",
    "                            None,\n",
    "                            shape])\n",
    "            i = neStarts[i].end - 1\n",
    "        #If not a NE, add the word if it's not a stopword or a non-alpha (not regular letters)\n",
    "        else:\n",
    "            if (doc[i].is_stop == False and doc[i].is_alpha == True):\n",
    "                word = doc[i]\n",
    "\n",
    "                currentSentence = getSentenceForWordPosition(i, senStarts)\n",
    "                wordLen = 1\n",
    "\n",
    "                newWords.append([word.text,\n",
    "                                0,\n",
    "                                0,\n",
    "                                currentSentence,\n",
    "                                wordLen,\n",
    "                                None,\n",
    "                                word.pos_,\n",
    "                                word.tag_,\n",
    "                                word.dep_,\n",
    "                                word.shape_])\n",
    "        i += 1\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading the text for which we want to label the words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_json('../data/squad-v1/train-v1.1.json', orient='column')\n",
    "dev = pd.read_json('../data/squad-v1/dev-v1.1.json', orient='column')\n",
    "\n",
    "df = pd.concat([train, dev], ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Architecturally, the school has a Catholic character. Atop the Main Building\\'s gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend \"Venite Ad Me Omnes\". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "titleId = 0\n",
    "paragraphId = 0\n",
    "\n",
    "text = df['data'][titleId]['paragraphs'][paragraphId]['context']\n",
    "text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = []\n",
    "addWordsForParagrapgh(words, text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>titleId</th>\n",
       "      <th>paragrapghId</th>\n",
       "      <th>sentenceId</th>\n",
       "      <th>wordCount</th>\n",
       "      <th>NER</th>\n",
       "      <th>POS</th>\n",
       "      <th>TAG</th>\n",
       "      <th>DEP</th>\n",
       "      <th>shape</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Architecturally</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>ADV</td>\n",
       "      <td>RB</td>\n",
       "      <td>advmod</td>\n",
       "      <td>Xxxxx</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>school</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>nsubj</td>\n",
       "      <td>xxxx</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Catholic</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>NORP</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Xxxxx</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>character</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>dobj</td>\n",
       "      <td>xxxx</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Atop</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>ADP</td>\n",
       "      <td>IN</td>\n",
       "      <td>prep</td>\n",
       "      <td>Xxxx</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              text  titleId  paragrapghId  sentenceId  wordCount   NER   POS  \\\n",
       "0  Architecturally        0             0         0.0          1  None   ADV   \n",
       "1           school        0             0         0.0          1  None  NOUN   \n",
       "2         Catholic        0             0         0.0          1  NORP  None   \n",
       "3        character        0             0         0.0          1  None  NOUN   \n",
       "4             Atop        0             0         1.0          1  None   ADP   \n",
       "\n",
       "    TAG     DEP   shape  \n",
       "0    RB  advmod   Xxxxx  \n",
       "1    NN   nsubj    xxxx  \n",
       "2  None    None   Xxxxx  \n",
       "3    NN    dobj    xxxx  \n",
       "4    IN    prep    Xxxx  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordColums = ['text', 'titleId', 'paragrapghId', 'sentenceId','wordCount', 'NER', 'POS', 'TAG', 'DEP','shape']\n",
    "df = pd.DataFrame(words, columns=wordColums)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## One-hot encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NER\n",
      "POS\n",
      "TAG\n",
      "DEP\n"
     ]
    }
   ],
   "source": [
    "columnsToEncode = ['NER', 'POS', \"TAG\", 'DEP']\n",
    "\n",
    "for column in columnsToEncode:\n",
    "    print(column)\n",
    "    one_hot = pd.get_dummies(df[column])\n",
    "    one_hot = one_hot.add_prefix(column + '_')\n",
    "\n",
    "    df = df.drop(column, axis = 1)\n",
    "    df = df.join(one_hot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>titleId</th>\n",
       "      <th>paragrapghId</th>\n",
       "      <th>sentenceId</th>\n",
       "      <th>wordCount</th>\n",
       "      <th>shape</th>\n",
       "      <th>NER_CARDINAL</th>\n",
       "      <th>NER_DATE</th>\n",
       "      <th>NER_FAC</th>\n",
       "      <th>NER_GPE</th>\n",
       "      <th>...</th>\n",
       "      <th>DEP_appos</th>\n",
       "      <th>DEP_attr</th>\n",
       "      <th>DEP_compound</th>\n",
       "      <th>DEP_conj</th>\n",
       "      <th>DEP_dobj</th>\n",
       "      <th>DEP_nsubj</th>\n",
       "      <th>DEP_pobj</th>\n",
       "      <th>DEP_poss</th>\n",
       "      <th>DEP_prep</th>\n",
       "      <th>DEP_relcl</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Architecturally</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>school</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>xxxx</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Catholic</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>character</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>xxxx</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Atop</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Xxxx</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 43 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              text  titleId  paragrapghId  sentenceId  wordCount   shape  \\\n",
       "0  Architecturally        0             0         0.0          1   Xxxxx   \n",
       "1           school        0             0         0.0          1    xxxx   \n",
       "2         Catholic        0             0         0.0          1   Xxxxx   \n",
       "3        character        0             0         0.0          1    xxxx   \n",
       "4             Atop        0             0         1.0          1    Xxxx   \n",
       "\n",
       "   NER_CARDINAL  NER_DATE  NER_FAC  NER_GPE  ...  DEP_appos  DEP_attr  \\\n",
       "0             0         0        0        0  ...          0         0   \n",
       "1             0         0        0        0  ...          0         0   \n",
       "2             0         0        0        0  ...          0         0   \n",
       "3             0         0        0        0  ...          0         0   \n",
       "4             0         0        0        0  ...          0         0   \n",
       "\n",
       "   DEP_compound  DEP_conj  DEP_dobj  DEP_nsubj  DEP_pobj  DEP_poss  DEP_prep  \\\n",
       "0             0         0         0          0         0         0         0   \n",
       "1             0         0         0          1         0         0         0   \n",
       "2             0         0         0          0         0         0         0   \n",
       "3             0         0         1          0         0         0         0   \n",
       "4             0         0         0          0         0         0         1   \n",
       "\n",
       "   DEP_relcl  \n",
       "0          0  \n",
       "1          0  \n",
       "2          0  \n",
       "3          0  \n",
       "4          0  \n",
       "\n",
       "[5 rows x 43 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dammit! One-hot encoding gave me more columns on the full sample. I need to add the rest of the columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictorColumns = ['wordCount','NER_CARDINAL','NER_DATE','NER_EVENT','NER_FAC','NER_GPE','NER_LANGUAGE','NER_LAW','NER_LOC','NER_MONEY','NER_NORP','NER_ORDINAL','NER_ORG','NER_PERCENT','NER_PERSON','NER_PRODUCT','NER_QUANTITY','NER_TIME','NER_WORK_OF_ART','POS_ADJ','POS_ADP','POS_ADV','POS_CCONJ','POS_DET','POS_INTJ','POS_NOUN','POS_NUM','POS_PART','POS_PRON','POS_PROPN','POS_PUNCT','POS_SYM','POS_VERB','POS_X','TAG_''','TAG_-LRB-','TAG_.','TAG_ADD','TAG_AFX','TAG_CC','TAG_CD','TAG_DT','TAG_EX','TAG_FW','TAG_IN','TAG_JJ','TAG_JJR','TAG_JJS','TAG_LS','TAG_MD','TAG_NFP','TAG_NN','TAG_NNP','TAG_NNPS','TAG_NNS','TAG_PDT','TAG_POS','TAG_PRP','TAG_PRP$','TAG_RB','TAG_RBR','TAG_RBS','TAG_RP','TAG_SYM','TAG_TO','TAG_UH','TAG_VB','TAG_VBD','TAG_VBG','TAG_VBN','TAG_VBP','TAG_VBZ','TAG_WDT','TAG_WP','TAG_WRB','TAG_XX','DEP_ROOT','DEP_acl','DEP_acomp','DEP_advcl','DEP_advmod','DEP_agent','DEP_amod','DEP_appos','DEP_attr','DEP_aux','DEP_auxpass','DEP_case','DEP_cc','DEP_ccomp','DEP_compound','DEP_conj','DEP_csubj','DEP_csubjpass','DEP_dative','DEP_dep','DEP_det','DEP_dobj','DEP_expl','DEP_intj','DEP_mark','DEP_meta','DEP_neg','DEP_nmod','DEP_npadvmod','DEP_nsubj','DEP_nsubjpass','DEP_nummod','DEP_oprd','DEP_parataxis','DEP_pcomp','DEP_pobj','DEP_poss','DEP_preconj','DEP_predet','DEP_prep','DEP_prt','DEP_punct','DEP_quantmod','DEP_relcl','DEP_xcomp']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "wordsDf = pd.DataFrame(columns=predictorColumns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wordCount</th>\n",
       "      <th>NER_CARDINAL</th>\n",
       "      <th>NER_DATE</th>\n",
       "      <th>NER_EVENT</th>\n",
       "      <th>NER_FAC</th>\n",
       "      <th>NER_GPE</th>\n",
       "      <th>NER_LANGUAGE</th>\n",
       "      <th>NER_LAW</th>\n",
       "      <th>NER_LOC</th>\n",
       "      <th>NER_MONEY</th>\n",
       "      <th>...</th>\n",
       "      <th>DEP_pobj</th>\n",
       "      <th>DEP_poss</th>\n",
       "      <th>DEP_preconj</th>\n",
       "      <th>DEP_predet</th>\n",
       "      <th>DEP_prep</th>\n",
       "      <th>DEP_prt</th>\n",
       "      <th>DEP_punct</th>\n",
       "      <th>DEP_quantmod</th>\n",
       "      <th>DEP_relcl</th>\n",
       "      <th>DEP_xcomp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>0 rows × 121 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [wordCount, NER_CARDINAL, NER_DATE, NER_EVENT, NER_FAC, NER_GPE, NER_LANGUAGE, NER_LAW, NER_LOC, NER_MONEY, NER_NORP, NER_ORDINAL, NER_ORG, NER_PERCENT, NER_PERSON, NER_PRODUCT, NER_QUANTITY, NER_TIME, NER_WORK_OF_ART, POS_ADJ, POS_ADP, POS_ADV, POS_CCONJ, POS_DET, POS_INTJ, POS_NOUN, POS_NUM, POS_PART, POS_PRON, POS_PROPN, POS_PUNCT, POS_SYM, POS_VERB, POS_X, TAG_, TAG_-LRB-, TAG_., TAG_ADD, TAG_AFX, TAG_CC, TAG_CD, TAG_DT, TAG_EX, TAG_FW, TAG_IN, TAG_JJ, TAG_JJR, TAG_JJS, TAG_LS, TAG_MD, TAG_NFP, TAG_NN, TAG_NNP, TAG_NNPS, TAG_NNS, TAG_PDT, TAG_POS, TAG_PRP, TAG_PRP$, TAG_RB, TAG_RBR, TAG_RBS, TAG_RP, TAG_SYM, TAG_TO, TAG_UH, TAG_VB, TAG_VBD, TAG_VBG, TAG_VBN, TAG_VBP, TAG_VBZ, TAG_WDT, TAG_WP, TAG_WRB, TAG_XX, DEP_ROOT, DEP_acl, DEP_acomp, DEP_advcl, DEP_advmod, DEP_agent, DEP_amod, DEP_appos, DEP_attr, DEP_aux, DEP_auxpass, DEP_case, DEP_cc, DEP_ccomp, DEP_compound, DEP_conj, DEP_csubj, DEP_csubjpass, DEP_dative, DEP_dep, DEP_det, DEP_dobj, DEP_expl, DEP_intj, ...]\n",
       "Index: []\n",
       "\n",
       "[0 rows x 121 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordsDf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "for column in wordsDf.columns:\n",
    "    if (column in df.columns):\n",
    "        wordsDf[column] = df[column]\n",
    "    else:\n",
    "        wordsDf[column] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wordCount</th>\n",
       "      <th>NER_CARDINAL</th>\n",
       "      <th>NER_DATE</th>\n",
       "      <th>NER_EVENT</th>\n",
       "      <th>NER_FAC</th>\n",
       "      <th>NER_GPE</th>\n",
       "      <th>NER_LANGUAGE</th>\n",
       "      <th>NER_LAW</th>\n",
       "      <th>NER_LOC</th>\n",
       "      <th>NER_MONEY</th>\n",
       "      <th>...</th>\n",
       "      <th>DEP_pobj</th>\n",
       "      <th>DEP_poss</th>\n",
       "      <th>DEP_preconj</th>\n",
       "      <th>DEP_predet</th>\n",
       "      <th>DEP_prep</th>\n",
       "      <th>DEP_prt</th>\n",
       "      <th>DEP_punct</th>\n",
       "      <th>DEP_quantmod</th>\n",
       "      <th>DEP_relcl</th>\n",
       "      <th>DEP_xcomp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 121 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   wordCount  NER_CARDINAL  NER_DATE  NER_EVENT  NER_FAC  NER_GPE  \\\n",
       "0          1             0         0          0        0        0   \n",
       "1          1             0         0          0        0        0   \n",
       "2          1             0         0          0        0        0   \n",
       "3          1             0         0          0        0        0   \n",
       "4          1             0         0          0        0        0   \n",
       "\n",
       "   NER_LANGUAGE  NER_LAW  NER_LOC  NER_MONEY  ...  DEP_pobj  DEP_poss  \\\n",
       "0             0        0        0          0  ...         0         0   \n",
       "1             0        0        0          0  ...         0         0   \n",
       "2             0        0        0          0  ...         0         0   \n",
       "3             0        0        0          0  ...         0         0   \n",
       "4             0        0        0          0  ...         0         0   \n",
       "\n",
       "   DEP_preconj  DEP_predet  DEP_prep  DEP_prt  DEP_punct  DEP_quantmod  \\\n",
       "0            0           0         0        0          0             0   \n",
       "1            0           0         0        0          0             0   \n",
       "2            0           0         0        0          0             0   \n",
       "3            0           0         0        0          0             0   \n",
       "4            0           0         1        0          0             0   \n",
       "\n",
       "   DEP_relcl  DEP_xcomp  \n",
       "0          0          0  \n",
       "1          0          0  \n",
       "2          0          0  \n",
       "3          0          0  \n",
       "4          0          0  \n",
       "\n",
       "[5 rows x 121 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordsDf.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I can't believe this worked."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictorPickleName = '../data/pickles/nb-predictor.pkl'\n",
    "predictor = loadPickle(predictorPickleName)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = predictor.predict(wordsDf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False,  True,  True,  True, False,  True, False,  True,  True,\n",
       "        True,  True,  True, False,  True,  True,  True,  True,  True,\n",
       "        True, False,  True,  True,  True,  True, False,  True,  True,\n",
       "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "       False, False,  True,  True,  True,  True,  True,  True,  True,\n",
       "       False,  True,  True,  True,  True,  True,  True,  True,  True])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Moment of truth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F Architecturally\n",
      "T school\n",
      "T Catholic\n",
      "T character\n",
      "F Atop\n",
      "T Main\n",
      "F Building\n",
      "T gold\n",
      "T dome\n",
      "T golden\n",
      "T statue\n",
      "T the Virgin Mary\n",
      "F Immediately\n",
      "T the Main Building\n",
      "T facing\n",
      "T copper\n",
      "T statue\n",
      "T Christ\n",
      "T arms\n",
      "F upraised\n",
      "T legend\n",
      "T Venite Ad Me Omnes\n",
      "T the Main Building\n",
      "T the Basilica of the Sacred Heart\n",
      "F Immediately\n",
      "T basilica\n",
      "T Grotto\n",
      "T Marian\n",
      "T place\n",
      "T prayer\n",
      "T reflection\n",
      "T replica\n",
      "T grotto\n",
      "T Lourdes\n",
      "T France\n",
      "T the Virgin Mary\n",
      "F reputedly\n",
      "F appeared\n",
      "T Saint Bernadette Soubirous\n",
      "T 1858\n",
      "T end\n",
      "T main\n",
      "T drive\n",
      "T direct\n",
      "T line\n",
      "F connects\n",
      "T 3\n",
      "T statues\n",
      "T the Gold Dome\n",
      "T simple\n",
      "T modern\n",
      "T stone\n",
      "T statue\n",
      "T Mary\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(y_pred)):\n",
    "    if (y_pred[i]):   \n",
    "        print('T', df.iloc[i]['text'])\n",
    "    else:\n",
    "        print('F', df.iloc[i]['text'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Well, since most of the words are labeled as answers, let's see just the incorrect ones."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Architecturally\n",
      "Atop\n",
      "Building\n",
      "Immediately\n",
      "upraised\n",
      "Immediately\n",
      "reputedly\n",
      "appeared\n",
      "connects\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(y_pred)):\n",
    "    if (y_pred[i] == False):   \n",
    "        print(df.iloc[i]['text'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Wow! That's pretty great actually!\n",
    "\n",
    "It would be great if I get confidence of the word being an answer rather than just a binary classification."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's not spoil this magic moment, when I actually think that I classifiy all the appropriate words and work on the incorrect answer generation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
